from __future__ import division
from operator import itemgetter, attrgetter
import os
import sys
import math

# current fixed version OPTION1
# purpose of this program
# input: load the data
# output: compute the KL values

queryTermWithItsProbabilitiesDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom100KWithTheirTrueProbablityAndOurOwnModelPredictedProbablity1D_2D_GoodTuringProbabilityAdded20130429"
inputFileHandler = open(inputFileName,"r")
# ignore the head line
inputFileHandler.readline()

# At the same time, compute the KL divergence as well
currentGoldStandardValue = 0.0
currentKLValue_byProf1D = 0.0
currentKLValue_byProf2D = 0.0
currentKLValue_goodTuring = 0.0

for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    # index number and corresponding values:
    # index = 0 : queryTermItself
    # index = 1 : goldStandardRealProbability
    # index = 2 : ourProbabilityModel1D
    # index = 3 : ourProbabilityModel2D
    # index = 4 : goodTuring
    
    queryTerm = lineElements[0]
    realProbability = float( lineElements[1] )
    ourProbabilityModel1D = float( lineElements[2] )
    ourProbabilityModel2D = float( lineElements[3] )
    goodTuring = float( lineElements[4] )
    
    if queryTerm not in queryTermWithItsProbabilitiesDict:
        queryTermWithItsProbabilitiesDict[queryTerm] = (realProbability,ourProbabilityModel1D,ourProbabilityModel2D)
        currentGoldStandardValue += math.log(realProbability / realProbability) * realProbability
        currentKLValue_byProf1D += math.log(realProbability / ourProbabilityModel1D) * realProbability
        currentKLValue_byProf2D += math.log(realProbability / ourProbabilityModel2D) * realProbability
        currentKLValue_goodTuring += math.log(realProbability / goodTuring) * realProbability
        
    else:
        print "Unexpected Mark1"
        exit(1)

print "currentKLValue_byProf1D:",currentKLValue_byProf1D
print "currentKLValue_byProf2D:",currentKLValue_byProf2D
print "currentGoldStandardValue:",currentGoldStandardValue
print "currentKLValue_goodTuring:",currentKLValue_goodTuring

print "len(queryTermWithItsProbabilitiesDict):",len(queryTermWithItsProbabilitiesDict)
inputFileHandler.close()


'''
# current fixed version OPTION2
# purpose of this program
# input: load the file called: queryTermsFrom4KWithTheLatestProbabilitySettings1DProbabilityAdded_GoodTuringProbabilityAdded20130425(The probability has already been computed)
# output: KL values, easy computation of KL formula.

queryTermWithItsProbabilitiesDict = {}
inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/probabilityDistributionEstimationByProf/fourSetOfQueriesByProf20130410/smallBucketsEquallizationMethodByProf20130411/queryTermsFrom4KWithTheLatestProbabilitySettings1DProbabilityAdded_GoodTuringProbabilityAdded20130425"
inputFileHandler = open(inputFileName,"r")
# ignore the head line
inputFileHandler.readline()

# At the same time, compute the KL divergence as well
currentKLValue_byProf2D = 0.0
currentKLValue1 = 0.0
currentKLValue_point9 = 0.0
currentKLValue_point8 = 0.0
currentKLValue_point7 = 0.0
currentKLValue_point6 = 0.0
currentKLValue_point5 = 0.0
currentKLValue_point4 = 0.0
currentKLValue_point3 = 0.0
currentKLValue_point2 = 0.0
currentKLValue_point1 = 0.0
currentKLValue_byProf1D = 0.0
currentGoodTuringValue = 0.0


for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    # index number and corresponding values:
    # index = 0 : queryTermItself
    # index = 1 : goldStandardRealProbability
    # index = 2 : ourProbabilityModel2D
    # (NO USE) index = 3 : normalizedProbability
    # index = 4 : addOneUnnormalizedProbablity
    # (NO USE) index = 5 : addOneNormalizedProbablity
    # index = 6 : addPoint9UnnormalizedProbablity
    # index = 7 : addPoint8UnnormalizedProbablity
    # index = 8 : addPoint7UnnormalizedProbablity
    # index = 9 : addPoint6UnnormalizedProbablity
    # index = 10 : addPoint5UnnormalizedProbablity
    # index = 11 : addPoint4UnnormalizedProbablity
    # index = 12 : addPoint3UnnormalizedProbablity
    # index = 13 : addPoint2UnnormalizedProbablity
    # index = 14 : addPoint1UnnormalizedProbablity
    # index = 15 : ourProbabilityModel1D
    # index = 16 : goodTuringProbabilityEstimated(1D)
    
    
    queryTerm = lineElements[0]
    realProbability = float( lineElements[1] )
    unNormalizedEstimatedProbability1 = float( lineElements[2] )
    # normalize this probability is NOT right
    # normalizedEstimatedProbability1 = float( lineElements[3] )
    unNormalizedEstimatedProbability2 = float( lineElements[4] )
    # normalize this probability is NOT right
    # normalizedEstimatedProbability2 = float( lineElements[5] )
    unNormalizedEstimatedProbability_point9 = float( lineElements[6] )
    unNormalizedEstimatedProbability_point8 = float( lineElements[7] )
    unNormalizedEstimatedProbability_point7 = float( lineElements[8] )
    unNormalizedEstimatedProbability_point6 = float( lineElements[9] )
    unNormalizedEstimatedProbability_point5 = float( lineElements[10] )
    unNormalizedEstimatedProbability_point4 = float( lineElements[11] )
    unNormalizedEstimatedProbability_point3 = float( lineElements[12] )
    unNormalizedEstimatedProbability_point2 = float( lineElements[13] )
    unNormalizedEstimatedProbability_point1 = float( lineElements[14] )
    
    unNormalizedEstimatedProbabilityByProf1D = float( lineElements[15] )
    goodTuringProbability = float( lineElements[16] )
    
    if queryTerm not in queryTermWithItsProbabilitiesDict:
        queryTermWithItsProbabilitiesDict[queryTerm] = (realProbability,unNormalizedEstimatedProbability1,unNormalizedEstimatedProbability2,unNormalizedEstimatedProbability_point9,unNormalizedEstimatedProbability_point8,unNormalizedEstimatedProbability_point7,unNormalizedEstimatedProbability_point6,unNormalizedEstimatedProbability_point5,unNormalizedEstimatedProbability_point4,unNormalizedEstimatedProbability_point3,unNormalizedEstimatedProbability_point2,unNormalizedEstimatedProbability_point1,unNormalizedEstimatedProbabilityByProf1D,goodTuringProbability)
        # using the normalized Estimated Probability method 2, the value is 0.282136334867
        currentKLValue_byProf2D += math.log(realProbability / unNormalizedEstimatedProbability1) * realProbability
        currentKLValue1 += math.log(realProbability / unNormalizedEstimatedProbability2) * realProbability
        currentKLValue_point9 += math.log(realProbability / unNormalizedEstimatedProbability_point9) * realProbability
        currentKLValue_point8 += math.log(realProbability / unNormalizedEstimatedProbability_point8) * realProbability
        currentKLValue_point7 += math.log(realProbability / unNormalizedEstimatedProbability_point7) * realProbability
        currentKLValue_point6 += math.log(realProbability / unNormalizedEstimatedProbability_point6) * realProbability
        currentKLValue_point5 += math.log(realProbability / unNormalizedEstimatedProbability_point5) * realProbability
        currentKLValue_point4 += math.log(realProbability / unNormalizedEstimatedProbability_point4) * realProbability
        currentKLValue_point3 += math.log(realProbability / unNormalizedEstimatedProbability_point3) * realProbability
        currentKLValue_point2 += math.log(realProbability / unNormalizedEstimatedProbability_point2) * realProbability
        currentKLValue_point1 += math.log(realProbability / unNormalizedEstimatedProbability_point1) * realProbability
        currentKLValue_byProf1D += math.log(realProbability / unNormalizedEstimatedProbabilityByProf1D) * realProbability
        currentGoodTuringValue += math.log(realProbability / goodTuringProbability) * realProbability
        # using the normalized Estimated Probability method 1, the value is 0.815157397534
        # currentKLValue1 += math.log(realProbability / normalizedEstimatedProbability1) * realProbability
        # gold compared standard, the value will be 0
        # currentKLValue1 += math.log(realProbability / realProbability) * realProbability
    else:
        print "Unexpected Mark1"
        exit(1)

print "currentKLValue_byProf2D:",currentKLValue_byProf2D
print "currentKLValue1:",currentKLValue1
print "currentKLValue_point9:",currentKLValue_point9
print "currentKLValue_point8:",currentKLValue_point8
print "currentKLValue_point7:",currentKLValue_point7
print "currentKLValue_point6:",currentKLValue_point6
print "currentKLValue_point5:",currentKLValue_point5
print "currentKLValue_point4:",currentKLValue_point4
print "currentKLValue_point3:",currentKLValue_point3
print "currentKLValue_point2:",currentKLValue_point2
print "currentKLValue_point1:",currentKLValue_point1
print "currentKLValue_byProf1D:",currentKLValue_byProf1D
print "currentGoodTuringValue:",currentGoodTuringValue

print "len(queryTermWithItsProbabilitiesDict):",len(queryTermWithItsProbabilitiesDict)
inputFileHandler.close()
'''

